#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import requests
import random
import re
import time
from bs4 import BeautifulSoup


version = 'Test_0.0.0.2'
class TaoBao(object):
    """淘宝，对搜索结果进行数据提取

    """
    def __init__(self):
        self.search_url_hearder = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"}
        self.timeout = random.choice(range(80, 180))

    def func_req_search_view(self, keyword):
        """对关键字进行搜索，清洗数据

        :param keyword: 搜索关键字
        :return:
        """
        _search_url = 'https://s.taobao.com/search?'
        page = 0
        _page_limit = 1 # 总页数
        _page_size = 44 # 每页跨度
        while page < _page_limit:
            _num = page * _page_size
            _search_data = {'q':keyword, 'imgfile':'', 'js':'1', 'stats_click':'search_radio_all%3A1', 'ie':'utf8',
                           'sort':'sale-desc', 's':'%s' % _num}
            _req = requests.get(url = _search_url, params = _search_data, headers = self.search_url_hearder,
                               timeout = self.timeout, stream = True)
            _bs = BeautifulSoup(_req.content, "html5lib")
            _scripts = _bs.head.find_all('script')
            for _string in _scripts[-1].stripped_strings:
                _r = re.search('{"pageName":(.*?)\n', _string)
                if _r and _num == 0:
                    data = json.loads(_r.group().rstrip(';\n'))
                    _page_limit = int(data['mods']['sortbar']['data']['pager']['totalPage'])
                    _page_size = int(data['mods']['sortbar']['data']['pager']['pageSize'])
                    _total_count = int(data['mods']['sortbar']['data']['pager']['totalCount'])
                    print('页数：{}；总结果数：{}；每页跨度：{}；地址：{}'.format(_page_limit, _total_count, _page_size, _req.url))
                    self.func_result(data, page)
                elif _r:
                    data = json.loads(_r.group().rstrip(';\n'))
                    self.func_result(data, page)
                    time.sleep(4)
                else:
                    print('数据匹配失败，请重试！')
            page += 1
        else:
            print('-------Perfectly End-------')
            print('5秒后自动退出！')
            time.sleep(6)

    def func_result(self, data, page):
        """数据提取

        :param data: 存放数据的json
        :param page: 页码
        :return:
        """
        _item_info = data['mods']['itemlist']['data']['auctions'] # 商品数据
        for item in _item_info:
            nick = item['nick'] # 店铺名
            # user_id = int(item['user_id'])
            raw_title = item['raw_title'] # 商品标题
            view_price = item['view_price'] # 促销价的最小值，例：促销价 ¥ 79.90-99.00 取79.90
            view_fee = item['view_fee'] # 运费
            item_loc = item['item_loc'] # 发货地址
            view_sales = item['view_sales'] # 收货人数量
            comment_count = 0 if not item['comment_count'] else int(item['comment_count']) # 评论数
            detail_url = item['detail_url'] # 详情页的地址
            isTmall = item['shopcard']['isTmall'] # 是否天猫店
            insert_sql = ('INSERT INTO taobao_test.shop_info_test(nick, raw_title, view_price, view_fee, item_loc,'
                    'view_sales, comment_count, detail_url, isTmall) VALUES ({nick},{raw_title},{view_price},'
                    '{view_fee},{item_loc},{view_sales},{comment_count},{detail_url},{isTmall})').format(nick = nick,
                                                                          raw_title = raw_title,view_price = view_price,
                                                                          view_fee = view_fee,item_loc = item_loc,
                                                                          view_sales = view_sales,comment_count = comment_count,
                                                                          detail_url = detail_url,isTmall = isTmall)
            print(insert_sql)
        else:
            print('end page: {}'.format(page + 1))


if __name__ == '__main__':
    print('版本号：' + version)
    t = TaoBao()
    t.func_req_search_view(input('请输入一个关键字以搜索：'))